package inseniarlingua.lang.util

import groovy.transform.CompileStatic;
import groovy.transform.TypeCheckingMode;

import org.cyberneko.html.parsers.SAXParser;

//@CompileStatic
class HtmlFreqListReader {
	def html

	public HtmlFreqListReader(File file) {
		super()
		//println "url = $url"

		SAXParser parser = new SAXParser()

		// UTF-8 is encoded in the file and it's wrong.  Just ignore the char set and it will default to ISO-8859-1
		parser.setFeature("http://cyberneko.org/html/features/scanner/ignore-specified-charset", true)

		html = new XmlSlurper(parser).parse(file)
	}

	@CompileStatic(TypeCheckingMode.SKIP)
	List<Map<String,String>> getWords() {
		def tables = html.'**'.findAll{it.name() == 'TABLE'}
		def table = null
		for (t in tables) {
			if (t.TR.size() > 30) {
				table = t
				break;
			}
		}
		List<Map<String,String>> ret = []
		Map<Integer,String> indexMap = [0:'rank', 1:'word', 2:'pos', 3:'frequency']
		table.TR.each { tr ->
			int i = 0
			final row = [:]
			tr.TD.each {
				final key = indexMap[i]
				if (key) {
					row[key] = it.toString()
					i++
				}
			}
//			println row
			if (row.rank.isNumber()) {
				ret << row
			}
		}
		return ret;
	}

	/*
	@CompileStatic(TypeCheckingMode.SKIP)
	List<Map<String,String>> getWords(final table) {
		Map<Integer,String> indexMap = [0:'rank', 1:'word', 2:'pos', 3:'frequency']
		table.TR.each { tr ->
			int i = 0
			final row = [:]
			tr.TD.each {
				final key = indexMap[i]
				if (key) {
					row[key] = it
					i++
				}
			}
//			println row
		}
	}
	*/

	static void main(String [] args) {
		final file = new File("dictionaries/eng/1-grams-5000.html")
		final reader = new HtmlFreqListReader(file);
		final words = reader.getWords()
		words.each { println it }
	}
}
